{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example, we will show how to use ```simple_bind``` API. \n",
    "\n",
    "Note it is a low level API, by using low level API we are able to touch more details about MXNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "import numpy as np\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use a very simple 1 hidden layer BatchNorm fully connected MNIST network to demo how to use low level API.\n",
    "The network looks like:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"102pt\" height=\"611pt\"\n",
       " viewBox=\"0.00 0.00 102.00 611.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 607)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-607 98,-607 98,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<polygon fill=\"#8dd3c7\" stroke=\"black\" points=\"94,-58 -7.10543e-15,-58 -7.10543e-15,-0 94,-0 94,-58\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-25.3\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- fc1 -->\n",
       "<g id=\"node2\" class=\"node\"><title>fc1</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-167 -7.10543e-15,-167 -7.10543e-15,-109 94,-109 94,-167\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-126.8\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- fc1&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>fc1&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-98.5824C47,-85.2841 47,-70.632 47,-58.2967\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-108.887 42.5001,-98.887 47,-103.887 47.0001,-98.887 47.0001,-98.887 47.0001,-98.887 47,-103.887 51.5001,-98.8871 47,-108.887 47,-108.887\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-79.8\" font-family=\"Times,serif\" font-size=\"14.00\">784</text>\n",
       "</g>\n",
       "<!-- bn1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>bn1</title>\n",
       "<polygon fill=\"#bebada\" stroke=\"black\" points=\"94,-276 -7.10543e-15,-276 -7.10543e-15,-218 94,-218 94,-276\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-243.3\" font-family=\"Times,serif\" font-size=\"14.00\">BatchNorm</text>\n",
       "</g>\n",
       "<!-- bn1&#45;&gt;fc1 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>bn1&#45;&gt;fc1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-207.582C47,-194.284 47,-179.632 47,-167.297\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-217.887 42.5001,-207.887 47,-212.887 47.0001,-207.887 47.0001,-207.887 47.0001,-207.887 47,-212.887 51.5001,-207.887 47,-217.887 47,-217.887\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-188.8\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- act1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>act1</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-385 -7.10543e-15,-385 -7.10543e-15,-327 94,-327 94,-385\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-359.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- act1&#45;&gt;bn1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>act1&#45;&gt;bn1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-316.582C47,-303.284 47,-288.632 47,-276.297\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-326.887 42.5001,-316.887 47,-321.887 47.0001,-316.887 47.0001,-316.887 47.0001,-316.887 47,-321.887 51.5001,-316.887 47,-326.887 47,-326.887\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-297.8\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- fc2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>fc2</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-494 -7.10543e-15,-494 -7.10543e-15,-436 94,-436 94,-494\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-468.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-453.8\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "<!-- fc2&#45;&gt;act1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>fc2&#45;&gt;act1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-425.582C47,-412.284 47,-397.632 47,-385.297\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-435.887 42.5001,-425.887 47,-430.887 47.0001,-425.887 47.0001,-425.887 47.0001,-425.887 47,-430.887 51.5001,-425.887 47,-435.887 47,-435.887\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-406.8\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- softmax -->\n",
       "<g id=\"node6\" class=\"node\"><title>softmax</title>\n",
       "<polygon fill=\"#b3de69\" stroke=\"black\" points=\"94,-603 -7.10543e-15,-603 -7.10543e-15,-545 94,-545 94,-603\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-570.3\" font-family=\"Times,serif\" font-size=\"14.00\">Softmax</text>\n",
       "</g>\n",
       "<!-- softmax&#45;&gt;fc2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>softmax&#45;&gt;fc2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-534.582C47,-521.284 47,-506.632 47,-494.297\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-544.887 42.5001,-534.887 47,-539.887 47.0001,-534.887 47.0001,-534.887 47.0001,-534.887 47,-539.887 51.5001,-534.887 47,-544.887 47,-544.887\"/>\n",
       "<text text-anchor=\"middle\" x=\"54\" y=\"-515.8\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f9c55b02048>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can use mx.sym in short of mx.symbol\n",
    "data = mx.sym.Variable(\"data\")\n",
    "fc1 = mx.sym.FullyConnected(data=data, num_hidden=128, name=\"fc1\")\n",
    "bn1 = mx.sym.BatchNorm(data=fc1, name=\"bn1\")\n",
    "act1 = mx.sym.Activation(data=bn1, name=\"act1\", act_type=\"tanh\")\n",
    "fc2 = mx.sym.FullyConnected(data=act1, name=\"fc2\", num_hidden=10)\n",
    "softmax = mx.sym.Softmax(data=fc2, name=\"softmax\")\n",
    "# visualize the network\n",
    "batch_size = 100\n",
    "data_shape = (batch_size, 784)\n",
    "mx.viz.plot_network(softmax, shape={\"data\":data_shape})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can use ```simple_bind``` api to generate executor from symbol."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# context different to ```mx.model```, \n",
    "# In mx.model, we wrapped parameter server, but for a single executor, the context is only able to be ONE device\n",
    "# run on cpu\n",
    "ctx = mx.cpu()\n",
    "# run on gpu\n",
    "# ctx = mx.gpu()\n",
    "# run on third gpu\n",
    "# ctx = mx.gpu(2)\n",
    "executor = softmax.simple_bind(ctx=ctx, data=data_shape, grad_req='write')\n",
    "# The default ctx is CPU, data's shape is required and ```simple_bind``` will try to infer all other required \n",
    "# For MLP, the ```grad_req``` is write to, and for RNN it is different"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After generating executor, we get get data from executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get argument arrays\n",
    "arg_arrays = executor.arg_arrays\n",
    "# get grad arrays\n",
    "grad_arrays = executor.grad_arrays\n",
    "# get aux_states arrays. Note: currently only BatchNorm symbol has auxiliary states, which is moving_mean and moving_var\n",
    "aux_arrays = executor.aux_arrays\n",
    "# get outputs from executor\n",
    "output_arrays = executor.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sequence of arrays is in same sequence of symbol arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "args:  {'fc2_bias': <mxnet.ndarray.NDArray object at 0x7f9c5662ec50>, 'data': <mxnet.ndarray.NDArray object at 0x7f9c5662eb70>, 'fc1_weight': <mxnet.ndarray.NDArray object at 0x7f9c5662e9e8>, 'fc1_bias': <mxnet.ndarray.NDArray object at 0x7f9c5662ed30>, 'bn1_beta': <mxnet.ndarray.NDArray object at 0x7f9c5662ebe0>, 'softmax_label': <mxnet.ndarray.NDArray object at 0x7f9c5662edd8>, 'fc2_weight': <mxnet.ndarray.NDArray object at 0x7f9c5662ec18>, 'bn1_gamma': <mxnet.ndarray.NDArray object at 0x7f9c5662eda0>}\n",
      "--------------------\n",
      "grads:  {'fc2_bias': <mxnet.ndarray.NDArray object at 0x7f9c5662eeb8>, 'data': None, 'fc1_weight': <mxnet.ndarray.NDArray object at 0x7f9c5662ef28>, 'fc1_bias': <mxnet.ndarray.NDArray object at 0x7f9c5662ef60>, 'bn1_beta': <mxnet.ndarray.NDArray object at 0x7f9c5662efd0>, 'softmax_label': None, 'fc2_weight': <mxnet.ndarray.NDArray object at 0x7f9c5662ee80>, 'bn1_gamma': <mxnet.ndarray.NDArray object at 0x7f9c5662ef98>}\n",
      "--------------------\n",
      "aux_states:  {'bn1_moving_mean': <mxnet.ndarray.NDArray object at 0x7f9c5662ee48>, 'bn1_moving_var': <mxnet.ndarray.NDArray object at 0x7f9c5662eb38>}\n",
      "--------------------\n",
      "outputs:  {'softmax_output': <mxnet.ndarray.NDArray object at 0x7f9c94302208>}\n"
     ]
    }
   ],
   "source": [
    "args = dict(zip(softmax.list_arguments(), arg_arrays))\n",
    "grads = dict(zip(softmax.list_arguments(), grad_arrays))\n",
    "outputs = dict(zip(softmax.list_outputs(), output_arrays))\n",
    "aux_states = dict(zip(softmax.list_auxiliary_states(), aux_arrays))\n",
    "# we can print the args we have\n",
    "print(\"args: \", args)\n",
    "print(\"-\" * 20)\n",
    "print(\"grads: \", grads)\n",
    "print(\"-\" * 20)\n",
    "print(\"aux_states: \", aux_states)\n",
    "print(\"-\" * 20)\n",
    "print(\"outputs: \", outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next step is intilize weight. We can set weight directly by using ```mx.random``` or numpy ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# helper function\n",
    "def Init(key, arr):\n",
    "    if \"weight\" in key:\n",
    "        arr[:] = mx.random.uniform(-0.07, 0.07, arr.shape)\n",
    "        # or\n",
    "        # arr[:] = np.random.uniform(-0.07, 0.07, arr.shape)\n",
    "    elif \"gamma\" in key:\n",
    "        # for batch norm slope\n",
    "        arr[:] = 1.0\n",
    "    elif \"bias\" in key:\n",
    "        arr[:] = 0\n",
    "    elif \"beta\" in key:\n",
    "        # for batch norm bias\n",
    "        arr[:] = 0\n",
    "\n",
    "# Init args\n",
    "for key, arr in args.items():\n",
    "    Init(key, arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we can customize our update rule. Here for demo purpose, we make a simple update rule to show mxnet feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def SGD(key, weight, grad, lr=0.1, grad_norm=batch_size):\n",
    "    # key is key for weight, we can customize update rule\n",
    "    # weight is weight array\n",
    "    # grad is grad array\n",
    "    # lr is learning rate\n",
    "    # grad_norm is scalar to norm gradient, usually it is batch_size\n",
    "    norm = 1.0 / grad_norm\n",
    "    # here we can bias' learning rate 2 times larger than weight\n",
    "    if \"weight\" in key or \"gamma\" in key:\n",
    "        weight[:] -= lr * (grad * norm)\n",
    "    elif \"bias\" in key or \"beta\" in key:\n",
    "        weight[:] -= 2.0 * lr * (grad * norm)\n",
    "    else:\n",
    "        pass\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we will make a data iterator. We can either use build in iterator to load from binary file or build a numpy iterator.\n",
    "\n",
    "For special case, you are free to write your own iterator in python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# We use utils function in sklearn to get MNIST dataset in pickle\n",
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.utils import shuffle\n",
    "mnist = fetch_mldata('MNIST original', data_home=\"./data\")\n",
    "# shuffle data\n",
    "X, y = shuffle(mnist.data, mnist.target)\n",
    "# split dataset\n",
    "train_data = X[:50000, :].astype('float32')\n",
    "train_label = y[:50000]\n",
    "val_data = X[50000: 60000, :].astype('float32')\n",
    "val_label = y[50000:60000]\n",
    "# Normalize data\n",
    "train_data[:] /= 256.0\n",
    "val_data[:] /= 256.0\n",
    "# Build iterator\n",
    "train_iter = mx.io.NDArrayIter(data=train_data, label=train_label, batch_size=batch_size, shuffle=True)\n",
    "val_iter = mx.io.NDArrayIter(data=val_data, label=val_label, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also, we need to define a helper function to calculate accuracy of current training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Accuracy(label, pred_prob):\n",
    "    pred = np.argmax(pred_prob, axis=1)\n",
    "    return np.sum(label == pred) * 1.0 / label.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we can train the network by using executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Finish training iteration 0\n",
      "INFO:root:Train Acc: 0.8888\n",
      "INFO:root:Val Acc: 0.9218\n",
      "INFO:root:Finish training iteration 1\n",
      "INFO:root:Train Acc: 0.9285\n",
      "INFO:root:Val Acc: 0.9355\n",
      "INFO:root:Finish training iteration 2\n",
      "INFO:root:Train Acc: 0.9421\n",
      "INFO:root:Val Acc: 0.9425\n"
     ]
    }
   ],
   "source": [
    "num_round = 3\n",
    "keys = softmax.list_arguments()\n",
    "# we use extra ndarray to save output of net\n",
    "pred_prob = mx.nd.zeros(executor.outputs[0].shape)\n",
    "for i in range(num_round):\n",
    "    train_iter.reset()\n",
    "    val_iter.reset()\n",
    "    train_acc = 0.\n",
    "    val_acc = 0.\n",
    "    nbatch = 0.\n",
    "    # train\n",
    "    for data, label in train_iter:\n",
    "        # copy data into args\n",
    "        args[\"data\"][:] = data # or we can ```data.copyto(args[\"data\"])```\n",
    "        args[\"softmax_label\"][:] = label\n",
    "        executor.forward(is_train=True)\n",
    "        pred_prob[:] = executor.outputs[0]\n",
    "        executor.backward()\n",
    "        for key in keys:\n",
    "            SGD(key, args[key], grads[key])\n",
    "        train_acc += Accuracy(label.asnumpy(), pred_prob.asnumpy())\n",
    "        nbatch += 1.\n",
    "    logging.info(\"Finish training iteration %d\" % i)\n",
    "    train_acc /= nbatch\n",
    "    nbatch = 0.\n",
    "    # eval\n",
    "    for data, label in val_iter:\n",
    "        args[\"data\"][:] = data\n",
    "        executor.forward(is_train=False)\n",
    "        pred_prob[:] = executor.outputs[0]\n",
    "        val_acc += Accuracy(label.asnumpy(), pred_prob.asnumpy())\n",
    "        nbatch += 1.\n",
    "    val_acc /= nbatch\n",
    "    logging.info(\"Train Acc: %.4f\" % train_acc)\n",
    "    logging.info(\"Val Acc: %.4f\" % val_acc)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "This is simple example gives a demo on how to directly use symbolic API to build a neural net from scratch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
